{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run NB03-EDA-MetaData-Check.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Columns where ‘Count of Unique Values Per Column’ is 2\n",
    "for column_name in df_columns_number_of_unique_values:\n",
    "    if df_columns_number_of_unique_values[column_name] == 2:\n",
    "        print (column_name, 'dtype: ', df[column_name].dtype)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dichotomous? (AKA binary / boolean)\n",
    "Columns where ‘Count of Unique Values Per Column’ is 2\n",
    "\n",
    "### Dichotomous:\n",
    "\n",
    "FLAG_CONT_MOBILE  \n",
    "FLAG_DOCUMENT_2  \n",
    "FLAG_DOCUMENT_3  \n",
    "FLAG_DOCUMENT_4  \n",
    "FLAG_DOCUMENT_5  \n",
    "FLAG_DOCUMENT_6  \n",
    "FLAG_DOCUMENT_7  \n",
    "FLAG_DOCUMENT_8  \n",
    "FLAG_DOCUMENT_9  \n",
    "FLAG_DOCUMENT_10  \n",
    "FLAG_DOCUMENT_11  \n",
    "FLAG_DOCUMENT_12  \n",
    "FLAG_DOCUMENT_13  \n",
    "FLAG_DOCUMENT_14  \n",
    "FLAG_DOCUMENT_15  \n",
    "FLAG_DOCUMENT_16  \n",
    "FLAG_DOCUMENT_17  \n",
    "FLAG_DOCUMENT_18  \n",
    "FLAG_DOCUMENT_19  \n",
    "FLAG_DOCUMENT_20  \n",
    "FLAG_DOCUMENT_21  \n",
    "FLAG_EMAIL  \n",
    "FLAG_EMP_PHONE  \n",
    "FLAG_MOBIL  \n",
    "FLAG_OWN_CAR  \n",
    "FLAG_OWN_REALTY  \n",
    "FLAG_PHONE  \n",
    "FLAG_WORK_PHONE  \n",
    "LIVE_CITY_NOT_WORK_CITY  \n",
    "LIVE_REGION_NOT_WORK_REGION  \n",
    "NAME_CONTRACT_TYPE  \n",
    "REG_CITY_NOT_LIVE_CITY  \n",
    "REG_CITY_NOT_WORK_CITY  \n",
    "REG_REGION_NOT_LIVE_REGION  \n",
    "REG_REGION_NOT_WORK_REGION  \n",
    "\n",
    "### Polychotomous\n",
    "EMERGENCYSTATE_MODE  \n",
    "\n",
    "### Non-Model\n",
    "TARGET  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_dichotomous_column_names = [\n",
    "'FLAG_CONT_MOBILE',\n",
    "'FLAG_DOCUMENT_2',\n",
    "'FLAG_DOCUMENT_3',\n",
    "'FLAG_DOCUMENT_4',\n",
    "'FLAG_DOCUMENT_5',\n",
    "'FLAG_DOCUMENT_6',\n",
    "'FLAG_DOCUMENT_7',\n",
    "'FLAG_DOCUMENT_8',\n",
    "'FLAG_DOCUMENT_9',\n",
    "'FLAG_DOCUMENT_10',\n",
    "'FLAG_DOCUMENT_11',\n",
    "'FLAG_DOCUMENT_12',\n",
    "'FLAG_DOCUMENT_13',\n",
    "'FLAG_DOCUMENT_14',\n",
    "'FLAG_DOCUMENT_15',\n",
    "'FLAG_DOCUMENT_16',\n",
    "'FLAG_DOCUMENT_17',\n",
    "'FLAG_DOCUMENT_18',\n",
    "'FLAG_DOCUMENT_19',\n",
    "'FLAG_DOCUMENT_20',\n",
    "'FLAG_DOCUMENT_21',\n",
    "'FLAG_EMAIL',\n",
    "'FLAG_EMP_PHONE',\n",
    "'FLAG_MOBIL',\n",
    "'FLAG_OWN_CAR',\n",
    "'FLAG_OWN_REALTY',\n",
    "'FLAG_PHONE',\n",
    "'FLAG_WORK_PHONE',\n",
    "'LIVE_CITY_NOT_WORK_CITY',\n",
    "'LIVE_REGION_NOT_WORK_REGION',\n",
    "'NAME_CONTRACT_TYPE',\n",
    "'REG_CITY_NOT_LIVE_CITY',\n",
    "'REG_CITY_NOT_WORK_CITY',\n",
    "'REG_REGION_NOT_LIVE_REGION',\n",
    "'REG_REGION_NOT_WORK_REGION'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for column_name in df_dichotomous_column_names:\n",
    "    if df_columns_number_of_unique_values[column_name] == 2:\n",
    "        print(df[column_name].value_counts(dropna=False, sort=True))\n",
    "        print(df[column_name].value_counts(dropna=False, normalize=True, sort=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NOT 0/1\n",
    "\n",
    "### FLAG_OWN_CAR\n",
    "N    202924 (0.659892)  \n",
    "Y    104587 (0.340108)  \n",
    "\n",
    "### FLAG_OWN_REALTY\n",
    "Y    213312 (0.693673)  \n",
    "N     94199 (0.306327)  \n",
    "\n",
    "### NAME_CONTRACT_TYPE\n",
    "Cash loans         278232 (0.904787)  \n",
    "Revolving loans     29279 (0.095213)  \n",
    "NOTE: This is to be *converted* to a boolean, a la dummy coded, with \"Cash loans\" as the reference category."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['FLAG_OWN_CAR'] = pd.Series(np.where(df['FLAG_OWN_CAR'].values == 'Y', 1, 0), df.index)\n",
    "df['FLAG_OWN_CAR'].value_counts(dropna=False, sort=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['FLAG_OWN_REALTY'] = pd.Series(np.where(df['FLAG_OWN_REALTY'].values == 'Y', 1, 0), df.index)\n",
    "df['FLAG_OWN_REALTY'].value_counts(dropna=False, sort=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df['NAME_CONTRACT_TYPE__Revolving_loans'] = pd.Series(np.where(df['NAME_CONTRACT_TYPE'].values == 'Revolving loans', 1, 0), df.index)\n",
    "#df.drop('NAME_CONTRACT_TYPE', axis=1)\n",
    "df.rename(columns={'NAME_CONTRACT_TYPE': 'NAME_CONTRACT_TYPE__Revolving_loans'}, inplace=True)\n",
    "#df = df.rename(columns={'NAME_CONTRACT_TYPE': 'NAME_CONTRACT_TYPE__Revolving_loans'})\n",
    "df['NAME_CONTRACT_TYPE__Revolving_loans'] = pd.Series(np.where(df['NAME_CONTRACT_TYPE__Revolving_loans'].values == 'Revolving loans', 1, 0), df.index)\n",
    "df['NAME_CONTRACT_TYPE__Revolving_loans'].value_counts(dropna=False, sort=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Columns where ‘Count of Unique Values Per Column’ is 3\n",
    "for column_name in df_columns_number_of_unique_values:\n",
    "    if df_columns_number_of_unique_values[column_name] == 3:\n",
    "        print (column_name, 'dtype: ', df[column_name].dtype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Columns where ‘Count of Unique Values Per Column’ is 3\n",
    "for column_name in df_columns_number_of_unique_values:\n",
    "    if df_columns_number_of_unique_values[column_name] == 3:\n",
    "        print(df[column_name].value_counts(dropna=False, sort=True))\n",
    "        print(df[column_name].value_counts(dropna=False, normalize=True, sort=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CODE_GENDER\n",
    "Looks like we should deal with those four records with a missing value place-holder and convert it to a boolean, with \"F\" as the reference level.\n",
    "\n",
    "### HOUSETYPE_MODE\n",
    "Not sure what is going on with this one. A little EDA provided no indication of pattern to the missingness.  \n",
    "It seems likely that this is Missing Not At Random (MNAR) and, therefore, we should dig deeper.\n",
    "For now, moving forward with replacing the missing values with \"MISSING\" and treating it as \"Unknown/Not Provided\".  \n",
    "\n",
    "### REGION_RATING_CLIENT\n",
    "This should be treated as a Categorical - Ordinal variable.\n",
    "Check this relationship between this and REGION_RATING_CLIENT_W_CITY.\n",
    "\n",
    "### REGION_RATING_CLIENT_W_CITY\n",
    "This should be treated as a Categorical - Ordinal variable.\n",
    "Check this relationship between this and REGION_RATING_CLIENT."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CODE_GENDER"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.loc[df.SK_ID_CURR == 141289, 'CODE_GENDER'] = 'F'\n",
    "df.loc[df.SK_ID_CURR == 319880, 'CODE_GENDER'] = 'F'\n",
    "df.loc[df.SK_ID_CURR == 196708, 'CODE_GENDER'] = 'F'\n",
    "df.loc[df.SK_ID_CURR == 144669, 'CODE_GENDER'] = 'M'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# HOUSETYPE_MODE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# REGION_RATING_CLIENT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# REGION_RATING_CLIENT_W_CITY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[(df['FLAG_OWN_CAR'] == 0) & (df['OWN_CAR_AGE'].notnull())]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[(df['FLAG_OWN_CAR'] == 1) & (df['OWN_CAR_AGE'].isnull())]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[(df['FLAG_OWN_REALTY'] == 0) & (df['HOUSETYPE_MODE'].notnull())]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[(df['FLAG_OWN_REALTY'] == 1) & (df['HOUSETYPE_MODE'].isnull())]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Correlation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(df['EXT_SOURCE_1'].corr(df['EXT_SOURCE_2'], method='pearson'))\n",
    "print(df['EXT_SOURCE_1'].corr(df['EXT_SOURCE_2'], method='spearman'))\n",
    "print(df['EXT_SOURCE_1'].corr(df['EXT_SOURCE_2'], method='kendall'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(df['EXT_SOURCE_2'].corr(df['EXT_SOURCE_3'], method='pearson'))\n",
    "print(df['EXT_SOURCE_2'].corr(df['EXT_SOURCE_3'], method='spearman'))\n",
    "print(df['EXT_SOURCE_2'].corr(df['EXT_SOURCE_3'], method='kendall'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(df['EXT_SOURCE_3'].corr(df['EXT_SOURCE_1'], method='pearson'))\n",
    "print(df['EXT_SOURCE_3'].corr(df['EXT_SOURCE_1'], method='spearman'))\n",
    "print(df['EXT_SOURCE_3'].corr(df['EXT_SOURCE_1'], method='kendall'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(df['AMT_INCOME_TOTAL'].corr(df['EXT_SOURCE_1'], method='pearson'))\n",
    "print(df['AMT_INCOME_TOTAL'].corr(df['EXT_SOURCE_1'], method='spearman'))\n",
    "print(df['AMT_INCOME_TOTAL'].corr(df['EXT_SOURCE_1'], method='kendall'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(df['AMT_INCOME_TOTAL'].corr(df['EXT_SOURCE_2'], method='pearson'))\n",
    "print(df['AMT_INCOME_TOTAL'].corr(df['EXT_SOURCE_2'], method='spearman'))\n",
    "print(df['AMT_INCOME_TOTAL'].corr(df['EXT_SOURCE_2'], method='kendall'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(df['AMT_INCOME_TOTAL'].corr(df['EXT_SOURCE_3'], method='pearson'))\n",
    "print(df['AMT_INCOME_TOTAL'].corr(df['EXT_SOURCE_3'], method='spearman'))\n",
    "print(df['AMT_INCOME_TOTAL'].corr(df['EXT_SOURCE_3'], method='kendall'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp_df = df[['TARGET', 'EXT_SOURCE_1']]\n",
    "tmp_df = tmp_df.dropna()\n",
    "tmp_df['TARGET'] = pd.to_numeric(tmp_df['TARGET'], errors='coerce')\n",
    "print(tmp_df['TARGET'].dtype)\n",
    "tmp_df['EXT_SOURCE_1'] = pd.to_numeric(tmp_df['EXT_SOURCE_1'], errors='coerce')\n",
    "print(tmp_df['EXT_SOURCE_1'].dtype)\n",
    "print(pointbiserialr(tmp_df['TARGET'], tmp_df['EXT_SOURCE_1']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp_df = df[['TARGET', 'EXT_SOURCE_2']]\n",
    "tmp_df = tmp_df.dropna()\n",
    "pointbiserialr(tmp_df['TARGET'], tmp_df['EXT_SOURCE_2'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp_df = df[['TARGET', 'EXT_SOURCE_3']]\n",
    "tmp_df = tmp_df.dropna()\n",
    "pointbiserialr(tmp_df['TARGET'], tmp_df['EXT_SOURCE_3'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp_df = df[['TARGET', 'AMT_INCOME_TOTAL']]\n",
    "tmp_df = tmp_df.dropna()\n",
    "pointbiserialr(tmp_df['TARGET'], tmp_df['AMT_INCOME_TOTAL'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.boxplot(x=df['AMT_INCOME_TOTAL']);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sns.boxplot(x='TARGET', y='AMT_INCOME_TOTAL', data=df);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#az = sns.boxplot(x='TARGET', y='AMT_INCOME_TOTAL', data=df);\n",
    "#az = sns.swarmplot(x='TARGET', y='AMT_INCOME_TOTAL', data=df, color='0.25');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.violinplot(x='TARGET', y='AMT_INCOME_TOTAL', data=df, scale='count');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(df['AMT_INCOME_TOTAL'].min())\n",
    "print(df['AMT_INCOME_TOTAL'].max())\n",
    "print(df['AMT_INCOME_TOTAL'].mean())\n",
    "print(df['AMT_INCOME_TOTAL'].median())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(np.percentile(df['AMT_INCOME_TOTAL'], 10))\n",
    "print(np.percentile(df['AMT_INCOME_TOTAL'], 20))\n",
    "print(np.percentile(df['AMT_INCOME_TOTAL'], 30))\n",
    "print(np.percentile(df['AMT_INCOME_TOTAL'], 40))\n",
    "print(np.percentile(df['AMT_INCOME_TOTAL'], 50))\n",
    "print(np.percentile(df['AMT_INCOME_TOTAL'], 60))\n",
    "print(np.percentile(df['AMT_INCOME_TOTAL'], 70))\n",
    "print(np.percentile(df['AMT_INCOME_TOTAL'], 80))\n",
    "print(np.percentile(df['AMT_INCOME_TOTAL'], 90))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp_percentile_count_boundaries = list()\n",
    "tmp_percentile_count_boundaries.append(df['AMT_INCOME_TOTAL'].min() - 1)\n",
    "for iter in range(10,100,10):\n",
    "    tmp_percentile_count_boundaries.append(np.percentile(df['AMT_INCOME_TOTAL'], iter))\n",
    "tmp_percentile_count_boundaries.append(df['AMT_INCOME_TOTAL'].max() + 1)\n",
    "sorted(tmp_percentile_count_boundaries)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp_percentile_10th = np.percentile(df['AMT_INCOME_TOTAL'], 10)\n",
    "print(df[['AMT_INCOME_TOTAL']][(df['AMT_INCOME_TOTAL'] <= tmp_percentile_10th)].count())\n",
    "tmp_percentile_10th"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "column_name = 'AMT_INCOME_TOTAL'\n",
    "print(df[column_name][(df[column_name] >= tmp_percentile_count_boundaries[0]) & (df[column_name] <= tmp_percentile_count_boundaries[1])].count())\n",
    "print(df[column_name][(df[column_name] >= tmp_percentile_count_boundaries[1]) & (df[column_name] <= tmp_percentile_count_boundaries[2])].count())\n",
    "print(df[column_name][(df[column_name] >= tmp_percentile_count_boundaries[2]) & (df[column_name] <= tmp_percentile_count_boundaries[3])].count())\n",
    "print(df[column_name][(df[column_name] >= tmp_percentile_count_boundaries[3]) & (df[column_name] <= tmp_percentile_count_boundaries[4])].count())\n",
    "print(df[column_name][(df[column_name] >= tmp_percentile_count_boundaries[4]) & (df[column_name] <= tmp_percentile_count_boundaries[5])].count())\n",
    "print(df[column_name][(df[column_name] >= tmp_percentile_count_boundaries[5]) & (df[column_name] <= tmp_percentile_count_boundaries[6])].count())\n",
    "print(df[column_name][(df[column_name] >= tmp_percentile_count_boundaries[6]) & (df[column_name] <= tmp_percentile_count_boundaries[7])].count())\n",
    "print(df[column_name][(df[column_name] >= tmp_percentile_count_boundaries[7]) & (df[column_name] <= tmp_percentile_count_boundaries[8])].count())\n",
    "print(df[column_name][(df[column_name] >= tmp_percentile_count_boundaries[8]) & (df[column_name] <= tmp_percentile_count_boundaries[9])].count())\n",
    "print(df[column_name][(df[column_name] >= tmp_percentile_count_boundaries[9]) & (df[column_name] <= tmp_percentile_count_boundaries[10])].count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(np.percentile(df['AMT_INCOME_TOTAL'],  1))\n",
    "print(df[['AMT_INCOME_TOTAL']][(df['AMT_INCOME_TOTAL'] <= np.percentile(df['AMT_INCOME_TOTAL'],  1))].count()[0])\n",
    "print(np.percentile(df['AMT_INCOME_TOTAL'], 99))\n",
    "print(df[['AMT_INCOME_TOTAL']][(df['AMT_INCOME_TOTAL'] >= np.percentile(df['AMT_INCOME_TOTAL'], 99))].count()[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(np.percentile(df['AMT_INCOME_TOTAL'],  0.1))\n",
    "print(df[['AMT_INCOME_TOTAL']][(df['AMT_INCOME_TOTAL'] <= np.percentile(df['AMT_INCOME_TOTAL'],  0.1))].count()[0])\n",
    "print(np.percentile(df['AMT_INCOME_TOTAL'], 99.9))\n",
    "print(df[['AMT_INCOME_TOTAL']][(df['AMT_INCOME_TOTAL'] >= np.percentile(df['AMT_INCOME_TOTAL'], 99.9))].count()[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(np.percentile(df['AMT_INCOME_TOTAL'],  0.01))\n",
    "print(df[['AMT_INCOME_TOTAL']][(df['AMT_INCOME_TOTAL'] <= np.percentile(df['AMT_INCOME_TOTAL'],  0.01))].count()[0])\n",
    "print(np.percentile(df['AMT_INCOME_TOTAL'], 99.99))\n",
    "print(df[['AMT_INCOME_TOTAL']][(df['AMT_INCOME_TOTAL'] >= np.percentile(df['AMT_INCOME_TOTAL'], 99.99))].count()[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(np.percentile(df['AMT_INCOME_TOTAL'],  0.001))\n",
    "print(df[['AMT_INCOME_TOTAL']][(df['AMT_INCOME_TOTAL'] <= np.percentile(df['AMT_INCOME_TOTAL'],  0.001))].count()[0])\n",
    "print(np.percentile(df['AMT_INCOME_TOTAL'], 99.999))\n",
    "print(df[['AMT_INCOME_TOTAL']][(df['AMT_INCOME_TOTAL'] >= np.percentile(df['AMT_INCOME_TOTAL'], 99.999))].count()[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(np.percentile(df['AMT_INCOME_TOTAL'],  0.0001))\n",
    "print(df[['AMT_INCOME_TOTAL']][(df['AMT_INCOME_TOTAL'] <= np.percentile(df['AMT_INCOME_TOTAL'],  0.0001))].count()[0])\n",
    "print(np.percentile(df['AMT_INCOME_TOTAL'], 99.9999))\n",
    "print(df[['AMT_INCOME_TOTAL']][(df['AMT_INCOME_TOTAL'] >= np.percentile(df['AMT_INCOME_TOTAL'], 99.9999))].count()[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "column_name = 'AMT_INCOME_TOTAL'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[column_name].dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(df[column_name].min())\n",
    "print(df[column_name].max())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.histogram(df[column_name])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.hist(df[column_name])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.hist(df[column_name])\n",
    "plt.ticklabel_format(style='plain', axis='x')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.hist(df[column_name],\n",
    "         bins=[0,10000,100000,1000000,10000000,120000000])\n",
    "plt.ticklabel_format(style='plain', axis='x')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# An \"interface\" to matplotlib.axes.Axes.hist() method\n",
    "n, bins, patches = plt.hist(x=df[column_name],\n",
    "                            bins='auto',\n",
    "                            color='#0504aa',\n",
    "                            alpha=0.7,\n",
    "                            rwidth=0.85)\n",
    "plt.grid(axis='y', alpha=0.75)\n",
    "plt.xlabel('Value')\n",
    "plt.ylabel('Frequency')\n",
    "plt.title('Histogram of ' + column_name)\n",
    "#plt.text(23, 45, r'$\\mu=15, b=3$')\n",
    "maxfreq = n.max()\n",
    "# Set a clean upper y-axis limit.\n",
    "plt.ylim(ymax=np.ceil(maxfreq / 10) * 10 if maxfreq % 10 else maxfreq + 10)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# An \"interface\" to matplotlib.axes.Axes.hist() method\n",
    "n, bins, patches = plt.hist(x=df[column_name],\n",
    "                            bins=10,\n",
    "                            color='#0504aa',\n",
    "                            alpha=0.7,\n",
    "                            rwidth=0.85)\n",
    "plt.grid(axis='y', alpha=0.75)\n",
    "plt.xlabel('Value')\n",
    "plt.ylabel('Frequency')\n",
    "plt.title('Histogram of ' + column_name)\n",
    "#plt.text(23, 45, r'$\\mu=15, b=3$')\n",
    "maxfreq = n.max()\n",
    "# Set a clean upper y-axis limit.\n",
    "plt.ylim(ymax=np.ceil(maxfreq / 10) * 10 if maxfreq % 10 else maxfreq + 10)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.hist(df[column_name],\n",
    "         bins=30,\n",
    "         normed=True,\n",
    "         alpha=0.5,\n",
    "         histtype='stepfilled',\n",
    "         color='steelblue',\n",
    "         edgecolor='none')\n",
    "#plt.ticklabel_format(useOffset=False, style='plain', axis='both')\n",
    "#plt.ticklabel_format(style='plain', axis='both')\n",
    "#plt.ticklabel_format(style='plain', axis='x')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[column_name].plot.hist(grid=True,\n",
    "                           bins=20,\n",
    "                           rwidth=0.9,\n",
    "                           color='#607c8e')\n",
    "plt.title('Histogram of ' + column_name)\n",
    "plt.xlabel('Value')\n",
    "plt.ylabel('Frequency')\n",
    "plt.grid(axis='y', alpha=0.75)\n",
    "plt.ticklabel_format(style='plain', axis='x')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sp.stats.skew(df[column_name])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sp.stats.kurtosis(df[column_name])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[column_name].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[df[column_name] == 117000000.0]\n",
    "# SK_ID_CURR\n",
    "# 114967"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[[column_name]][df[column_name] != 117000000.0].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#plt.hist(df[[column_name]][df[column_name] != 117000000.0])\n",
    "#plt.ticklabel_format(style='plain', axis='x')\n",
    "#plt.show()\n",
    "#> ---------------------------------------------------------------------------\n",
    "#> AttributeError                            Traceback (most recent call last)\n",
    "#> ~\\Anaconda3\\lib\\site-packages\\matplotlib\\axes\\_base.py in ticklabel_format(self, **kwargs)\n",
    "#>    2567                 if axis == 'both' or axis == 'x':\n",
    "#> -> 2568                     self.xaxis.major.formatter.set_scientific(sb)\n",
    "#>    2569                 if axis == 'both' or axis == 'y':\n",
    "#> \n",
    "#> AttributeError: 'StrCategoryFormatter' object has no attribute 'set_scientific'\n",
    "#> \n",
    "#> During handling of the above exception, another exception occurred:\n",
    "#> \n",
    "#> AttributeError                            Traceback (most recent call last)\n",
    "#> <ipython-input-42-c0b04d2067b5> in <module>()\n",
    "#>       1 plt.hist(df[[column_name]][df[column_name] != 117000000.0])\n",
    "#> ----> 2 plt.ticklabel_format(style='plain', axis='x')\n",
    "#>       3 plt.show()\n",
    "#> \n",
    "#> ~\\Anaconda3\\lib\\site-packages\\matplotlib\\pyplot.py in ticklabel_format(**kwargs)\n",
    "#>    3749 @docstring.copy_dedent(Axes.ticklabel_format)\n",
    "#>    3750 def ticklabel_format(**kwargs):\n",
    "#> -> 3751     ret = gca().ticklabel_format(**kwargs)\n",
    "#>    3752     return ret\n",
    "#>    3753 \n",
    "#> \n",
    "#> ~\\Anaconda3\\lib\\site-packages\\matplotlib\\axes\\_base.py in ticklabel_format(self, **kwargs)\n",
    "#>    2591         except AttributeError:\n",
    "#>    2592             raise AttributeError(\n",
    "#> -> 2593                 \"This method only works with the ScalarFormatter.\")\n",
    "#>    2594 \n",
    "#>    2595     def locator_params(self, axis='both', tight=None, **kwargs):\n",
    "#> \n",
    "#> AttributeError: This method only works with the ScalarFormatter."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[df['SK_ID_CURR'] == '114967']['AMT_INCOME_TOTAL']\n",
    "#> 12840    117000000.0\n",
    "#> Name: AMT_INCOME_TOTAL, dtype: float64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.loc[df.SK_ID_CURR == '114967', 'AMT_INCOME_TOTAL'] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[df['SK_ID_CURR'] == '114967']['AMT_INCOME_TOTAL']\n",
    "#> 12840    0.0\n",
    "#> Name: AMT_INCOME_TOTAL, dtype: float64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.hist(df[column_name])\n",
    "plt.ticklabel_format(style='plain', axis='x')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[column_name].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[df[column_name] == 18000090.0]['SK_ID_CURR']\n",
    "#> 203693    336147\n",
    "#> Name: SK_ID_CURR, dtype: object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[df['SK_ID_CURR'] == '336147']['AMT_INCOME_TOTAL']\n",
    "#> 203693    18000090.0\n",
    "#> Name: AMT_INCOME_TOTAL, dtype: float64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.loc[df.SK_ID_CURR == '336147', 'AMT_INCOME_TOTAL'] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[df['SK_ID_CURR'] == '336147']['AMT_INCOME_TOTAL']\n",
    "#> 203693    0.0\n",
    "#> Name: AMT_INCOME_TOTAL, dtype: float64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.hist(df[column_name])\n",
    "plt.ticklabel_format(style='plain', axis='x')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[column_name].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[column_name].min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[column_name].median()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.hist(df[column_name],\n",
    "         bins=[0,25650,147150,1000000,120000000])\n",
    "plt.ticklabel_format(style='plain', axis='x')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.hist(df[column_name],\n",
    "         bins=[0,10000,20000,30000,40000,50000,60000,70000,80000,90000,\n",
    "               100000,110000,120000,130000,140000,150000,160000,170000,180000,190000,\n",
    "               200000,210000,220000,230000,240000,250000,260000,270000,280000,290000,\n",
    "               300000,310000,320000,330000,340000,350000,360000,370000,380000,390000,\n",
    "               400000,410000,420000,430000,440000,450000,460000,470000,480000,490000,\n",
    "               500000,510000,520000,530000,540000,550000,560000,570000,580000,590000,\n",
    "               600000,610000,620000,630000,640000,650000,660000,670000,680000,690000,\n",
    "               700000,710000,720000,730000,740000,750000,760000,770000,780000,790000,\n",
    "               800000,810000,820000,830000,840000,850000,860000,870000,880000,890000,\n",
    "               900000,910000,920000,930000,940000,950000,960000,970000,980000,990000,\n",
    "               1000000])\n",
    "plt.ticklabel_format(style='plain', axis='x')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['ratio__AMT_CREDIT__ANNUITY_RATIO'] = df['AMT_CREDIT'] / df['AMT_ANNUITY']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['ratio__AMT_INCOME_TOTAL__AMT_CREDIT'] = df['AMT_INCOME_TOTAL'] / df['AMT_CREDIT']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# How much of the price of the good was financed?\n",
    "df['ratio__AMT_CREDIT_AMT_GOODS_PRICE'] = df['AMT_CREDIT'] / df['AMT_GOODS_PRICE']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DAYS_BIRTH\n",
    "DAYS_EMPLOYED\n",
    "DAYS_ID_PUBLISH\n",
    "DAYS_LAST_PHONE_CHANGE\n",
    "DAYS_REGISTRATION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['ratio__DAYS_EMPLOYED__DAYS_BIRTH'] = df['DAYS_EMPLOYED'] / df['DAYS_BIRTH']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['ratio__DAYS_ID_PUBLISH__DAYS_BIRTH'] = df['DAYS_ID_PUBLISH'] / df['DAYS_BIRTH']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['ratio__DAYS_LAST_PHONE_CHANGE__DAYS_BIRTH'] = df['DAYS_LAST_PHONE_CHANGE'] / df['DAYS_BIRTH']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['ratio__DAYS_REGISTRATION__DAYS_BIRTH'] = df['DAYS_REGISTRATION'] / df['DAYS_BIRTH']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "flag_document_column_names = [\n",
    "'FLAG_DOCUMENT_2',\n",
    "'FLAG_DOCUMENT_3',\n",
    "'FLAG_DOCUMENT_4',\n",
    "'FLAG_DOCUMENT_5',\n",
    "'FLAG_DOCUMENT_6',\n",
    "'FLAG_DOCUMENT_7',\n",
    "'FLAG_DOCUMENT_8',\n",
    "'FLAG_DOCUMENT_9',\n",
    "'FLAG_DOCUMENT_10',\n",
    "'FLAG_DOCUMENT_11',\n",
    "'FLAG_DOCUMENT_12',\n",
    "'FLAG_DOCUMENT_13',\n",
    "'FLAG_DOCUMENT_14',\n",
    "'FLAG_DOCUMENT_15',\n",
    "'FLAG_DOCUMENT_16',\n",
    "'FLAG_DOCUMENT_17',\n",
    "'FLAG_DOCUMENT_18',\n",
    "'FLAG_DOCUMENT_19',\n",
    "'FLAG_DOCUMENT_20',\n",
    "'FLAG_DOCUMENT_21'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[flag_document_column_names].count(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[flag_document_column_names].sum(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[flag_document_column_names].dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[flag_document_column_names].astype('int').sum(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['FLAG_DOCUMENT__count'] = df[flag_document_column_names].astype('int').sum(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['FLAG_DOCUMENT__count']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "list(sorted(set(df['CNT_CHILDREN'].values)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['CNT_CHILDREN'].value_counts(ascending=True, dropna=False, sort=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['CNT_CHILDREN'].value_counts(ascending=True, dropna=False, normalize=True, sort=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DAYS...\n",
    "DAYS_BIRTH  \n",
    "DAYS_EMPLOYED  \n",
    "DAYS_ID_PUBLISH  \n",
    "DAYS_LAST_PHONE_CHANGE  \n",
    "DAYS_REGISTRATION  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['DAYS_BIRTH'].min() / 365.25 * -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['DAYS_BIRTH'].max() / 365.25 * -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['DAYS_BIRTH'].value_counts(ascending=True, dropna=False, sort=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['DAYS_BIRTH'].value_counts(ascending=True, dropna=False, normalize=True, sort=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['DAYS_BIRTH'].hist();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "HOUSETYPE_MODE\n",
    "FLAG_OWN_REALTY"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['HOUSETYPE_MODE'].value_counts(ascending=True, dropna=False, sort=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['FLAG_OWN_REALTY'].value_counts(ascending=True, dropna=False, sort=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.crosstab(df['FLAG_OWN_REALTY'], df['HOUSETYPE_MODE'], dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.crosstab(df['HOUSETYPE_MODE'], df['FLAG_OWN_REALTY'], dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import scipy.stats as stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_crosstab = pd.crosstab(index=df['FLAG_OWN_REALTY'], columns=df['HOUSETYPE_MODE'])\n",
    "print(df_crosstab)\n",
    "odds_ratio, p_value = stats.fisher_exact(df_crosstab)\n",
    "print(odds_ratio)\n",
    "print(p_value)\n",
    "chi_square_statistic, p_value, degrees_of_freedom, expected_frequencies = stats.chi2_contingency(df_crosstab)\n",
    "print(chi_square_statistic)\n",
    "print(p_value)\n",
    "print(degrees_of_freedom)\n",
    "print(expected_frequencies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['HOUSETYPE_MODE'].fillna('MISSING', inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Missing Values\n",
    "# Missing Values Place-Holders\n",
    "\n",
    "### \"XNA\"\n",
    "### 365243"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[df['CODE_GENDER'] == 'XNA']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[df['ORGANIZATION_TYPE'] == 'XNA']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[df['DAYS_EMPLOYED'] == 365243]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df[df['CODE_GENDER'] == 'XNA']\n",
    "#df[[['SK_ID_CURR','DAYS_EMPLOYED']],df['DAYS_EMPLOYED'] == 365243]\n",
    "df[['SK_ID_CURR','DAYS_EMPLOYED']df['DAYS_EMPLOYED'] == 365243]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df.loc[df['DAYS_EMPLOYED'] == 365243, 'SK_ID_CURR']\n",
    "df.loc[df['DAYS_EMPLOYED'] == 365243, ['SK_ID_CURR', 'DAYS_EMPLOYED']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "FLAG_OWN_CAR  \n",
    "OWN_CAR_AGE  \n",
    "\n",
    "FLAG_OWN_REALTY  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['FLAG_OWN_REALTY'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['FLAG_OWN_CAR'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['OWN_CAR_AGE'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['OWN_CAR_AGE'].isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.violinplot(y='OWN_CAR_AGE', data=df, scale='count');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.violinplot(x='TARGET', y='OWN_CAR_AGE', data=df, scale='count');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.boxplot(y=df['OWN_CAR_AGE']);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.boxplot(x='TARGET', y='OWN_CAR_AGE', data=df);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
